import re
import random
import sys
import time
import datetime
import json
import math
import pprint
import requests

from bs4 import BeautifulSoup
from selenium import webdriver
from pymongo import MongoClient
from multiprocessing import Process
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities

sys.setrecursionlimit(1000000)
net = sys.argv[1] if len(sys.argv) == 2 else 'localhost'

client = MongoClient(net, 27017, connect = False)

wukong_user_db = client['wukong_user_db']
wukong_user_coll = wukong_user_db['wukong_user_coll']


def new_driver():
    options = webdriver.ChromeOptions()

    options.add_argument('--headless')
    options.add_argument('--disable-gpu')
    options.add_argument('--disable-images')
    desired_capabilities = options.to_capabilities()

    driver = webdriver.Chrome(desired_capabilities = desired_capabilities, executable_path = '/Users/xuchaosheng/Workspace/qs-scrapy/chromedriver')
    return driver


def parse_following(uid, driver = None):
    if not driver:
        driver = new_driver()

    for type in ['fans', 'follower']:
        cursor = 0

        while True:
            url = 'https://www.wukong.com/wenda/web/my/' + type + '/list/brow/?cursor=' + str(cursor) + '&count=10&other_uid=' + str(uid)

            try:
                driver.get(url)
            except:
                break

            try:
                soup = BeautifulSoup(driver.page_source, 'lxml')
                resp = json.loads(soup.body.get_text())
            except:
                break

            if not 'data' in resp or not resp['data']:
                break

            for item in resp['data'][type + '_list']:
                if not wukong_user_coll.find_one({'user_id': item['user_id']}):
                    wukong_user_coll.insert_one(item)
                    print(item['uname'])

            if 'has_more' in resp['data'] and not resp['data']['has_more']:
                break

            cursor = resp['data']['cursor']

    for i in wukong_user_coll.aggregate([{'$sample': {'size': 1}}]):
        parse_following(i['user_id'], driver = driver)


if __name__ == '__main__':
    pool = []

    for i in wukong_user_coll.aggregate([{'$sample': {'size': 10}}]):
        p = Process(target = parse_following, args = (i['user_id'],))
        pool.append(p)

    for p in pool:
        p.start()

    for p in pool:
        p.join()
